#! /bin/bash
# Download OBS packages _service and .spec. Resulting:
#    $HOME/obsfiles/PROJECT/_pkglist
#    $HOME/obsfiles/PROJECT/PACKAGE.filelist
#    $HOME/obsfiles/PROJECT/PACKAGE.service
#    $HOME/obsfiles/PROJECT/PACKAGE.spec
#    $HOME/obsfiles/PROJECT/_log
#
# If any error or interrupt occurred, please check the message and _log file.
# Then prefer `osc api` command to correct them, as this tool will walk through
# all the packages whenever running.
# For 4000 packages with default workers, initial download costs about 6 hours,
# while thereafter be much lesser.
#
# To extract version from spec files, run
#    rpmspec -q --srpm --queryformat="%{NAME} %{VERSION}-%{RELEASE}\n" \
#           $HOME/obsfiles/PROJECT/*.spec 2>/dev/null
# For 4000 packages, it may take 1 hour.

function usage()
{
    cat << "EOF"
Download OBS packages _service and .spec, saving at $HOME/obsfiles/PROJECT/.

Usage: $0 [-o obs] [-n workers] [-h] project
           -o -- OBS alias/URL, default to oscrc config, see `osc -A`
           -n -- number of download processes, default 10
           -h -- help

EOF
}

# $1 -- spec/serv, $2 -- package name, $3 -- new filelist
# output: newname newmd5 oldname oldmd5
function get_name_md5()
{
    if [[ $1 == spec ]]; then
        key='"([^[:blank:]]+\.spec)" md5="([0-9a-fA-F]+)"' 
    else
        key='"(_service)" md5="([0-9a-f]+)"' 
    fi

    if [[ $3 =~ $key ]]; then
        echo ${BASH_REMATCH[1]} ${BASH_REMATCH[2]}
    else
        return
    fi

    if [[ -f $2.filelist ]]; then
        filelist=$(< $2.filelist)
        if [[ $filelist =~ $key ]]; then
            echo ${BASH_REMATCH[1]} ${BASH_REMATCH[2]}
        fi
    fi
}

# $1 -- package name, $2 -- OBS file name, $3 -- saved name
function download_file()
{
    osc $obs api -d expand /source/$proj/$1/$2 > "$3" ||\
        { wfailed+=1; echo "failed file: $3" >&2; }
}

function set_signaled()
{
    is_signaled=true
}

# $1 -- start index in pkgs[], $2 -- end index(exclusive), $3 -- temporary log
function worker_func()
{
    set +e
    local -i wadded=0
    local -i wupdated=0
    local -i wfailed=0

    # all parent unexpected info, go to temporary log through stderr
    >$3
    exec 2>>$3

    # to keep data consistency, using this to flag graceful exit 
    is_signaled=false
    trap set_signaled SIGTERM

    for (( i=$1 ; i<$2; i++ )); do
        p=${pkgs[i]}

        # graceful exit checkpoint 1
        if [[ $is_signaled == "true" ]]; then
            break
        fi
        newfilelist=`osc $obs api -d expand /source/$proj/$p`||{ wfailed+=3;\
            echo "download package filelist failed: $p" >&2; continue; }
        spec=(`get_name_md5 spec $p "$newfilelist"`)
        serv=(`get_name_md5 serv $p "$newfilelist"`)
        if [[ -z "${serv[*]}" || -z "${spec[*]}" ]]; then
            wfailed+=3
            echo "no .spec or _service file: $p" >&2
            continue;
        fi

        # graceful exit checkpoint 2
        if [[ "$is_signaled" == "true" ]]; then
            break
        fi
        if [[ ! -f ${p}.filelist || "${spec[1]}" != "${spec[3]}" ]]; then
            download_file $p ${spec[0]} ${p}.spec
            download_file $p ${serv[0]} ${p}.service
            if [[ -f ${p}.filelist ]]; then
                echo "update package: $p" >&2
                wupdated+=1
            else
                echo "add package: $p" >&2
                wadded+=1
            fi
        elif [[ "${serv[1]}" != "${serv[3]}" ]]; then
            download_file $p ${serv[0]} ${p}.service
            echo "update package: $p" >&2
            wupdated+=1
        else
            continue
        fi
        echo "$newfilelist" > ${p}.filelist
    done

    # only output these data to stdout ==> then pipe to parent
    echo $wadded $wupdated $wfailed
}

#   subprocess part
###########################################################################
#   parent part

function kill_workers()
{
    echo "interrupting..." >&2
    for (( i=0; i<$nproc; i++ )); do
        kill -SIGTERM ${worker_pid[i]} &>/dev/null
    done
    echo "INTERRUPTED"
}

function do_download()
{
    # caculate and dispatch works
    n=$(( ${#pkgs[@]} / nproc ))
    rem=$(( ${#pkgs[@]} % nproc ))
    local -i endidx=0
    echo "If you see 'warning: execute_coproc: ...', ignore them." >&2
    for (( i=0; i<$nproc; i++ )); do
        startidx=$endidx
        endidx=$(( startidx + n ))
        if (( i < rem )); then
            endidx+=1
        fi
        coproc worker_func $startidx $endidx _log.$i
        # duplicate FD to try avoid pipe broken due to worker exit
        exec {from_worker_fd[i]}>&${COPROC[0]}
        worker_pid[i]=$COPROC_PID
    done

    # enable keyboard interrupt
    trap kill_workers SIGINT SIGTERM
    wait
    
    # collect workers data
    for (( i=0; i<$nproc; i++ )); do
        read -a count <&${from_worker_fd[i]}
        added+=count[0]
        updated+=count[1]
        failed+=count[2]
        cat _log.$i
        rm -f _log.$i
        exec {from_worker_fd[i]}>&-
    done
}

# update _pkglist, fill in $pkgs, verify $nproc, check removed packages
function do_prepare()
{
    pkgs=($(< /tmp/_pkglist))
    if (( nproc > ${#pkgs[@]} )); then
        nproc=${#pkgs[@]}
    fi

    if [[ -f _pkglist ]]; then
        diff=`diff -uN _pkglist /tmp/_pkglist | grep -e '^-' | sed '1d'`
        for p in $diff; do
            echo "remove package: ${p:1}"
            removed+=1
            rm -f ${p:1}.*
        done
    fi
    mv -f /tmp/_pkglist .
}

set -e

# parse options
args=`getopt -u -o o:n:h -l help -- $@`
set -- $args
while [[ "$1" != "--" ]]; do
    case "$1" in
        -o)  obs=$2; shift 2;;
        -n)  nproc=$2
            if (( nproc <= 0 )); then
                echo "Bad workers number"
                exit 1
             fi
             shift 2;;
        -h|--help)  usage; exit 0;;
    esac
done
case $# in
    1)  echo "need project name"
        exit 2;;
    2)  ;;
    *)  echo "too many arguments"
        exit 3;;
esac

# global variables
obs=${obs:+"-A $obs"}
nproc=${nproc:-10}
proj=$2
declare -a worker_pid
declare -a from_worker_fd
declare -a pkgs
declare -i added=0
declare -i removed=0
declare -i updated=0
declare -i failed=0

echo "Begin time-consuming work..."
# test if $obs or $proj valid
set -o pipefail
osc $obs api /source/$proj | cut -s -f2 -d\" | sed '1d'| sort -s > /tmp/_pkglist
set +o pipefail

mkdir -p $HOME/obsfiles/$proj
pushd $HOME/obsfiles/$proj > /dev/null
exec >>_log
# ignore runtime error to tolarate network failure
set +e
# disable keyboard interrupt at first to try keep data consistency
trap "" SIGINT SIGTERM

echo "`date -Iseconds` begin"
do_prepare
do_download
echo "tatol added:$added removed:$removed updated:$updated ($failed files failed), cost $((SECONDS/60)) minutes" | tee /dev/stderr
echo "--------------------------------"

popd > /dev/null
